package com.undsf.study7816.fatego

import com.fasterxml.jackson.databind.ObjectMapper
import com.undsf.study7816.BaseCrawler
import com.undsf.study7816.fatego.po.fgowiki.ServantModel
import org.apache.logging.log4j.LogManager
import redis.clients.jedis.Jedis

/**
 * Created by Arathi on 2017/8/16.
 */
class FgowikiCrawler extends BaseCrawler {
    static baseUrl = "http://fgowiki.com"
    static jedis = new Jedis("localhost", 6379)
    static logger = LogManager.getLogger(FgowikiCrawler.class)

    FgowikiCrawler() {

    }

    @Override
    def crawl() {
        def idList = crawlList()
        cacheAllPage(idList)
        def servantList = convert(idList)
        for (def servant : servantList) {
            println(servant.toString())
        }
    }

    def crawlList() {
        return 1..117
    }

    def cacheAllPage(idList) {
        logger.info("开始抓取页面")
        for (def id : idList) {
            def url = baseUrl + "/guide/petdetail/" + id
            if (jedis.hexists("pages", url)) {
                continue
            }
            logger.info("crawling " + url + " ...")
            def html = sendGetRequest(url)
            jedis.hset("pages", url, html)
        }
        logger.info("页面抓取完成，已缓存到Redis")
    }

    // def crawlServants(idList) {
    //     logger.info("开始抽取json")
    //     for (def id : idList) {
    //         def featureStr = "var datadetail = "
    //         def url = baseUrl + "/guide/petdetail/" + id
    //         def html = sendGetRequest(url)
    //         def startAt = html.indexOf(featureStr)
    //         def endAt = html.indexOf("\n", startAt)
    //         def jsonStr = html.substring(startAt + featureStr.length(), endAt - 1)
    //         jedis.hset("servants", id+"", jsonStr)
    //     }
    //     logger.info("抽取json完成，已缓存到Redis")
    // }

    def convert(idList) {
        def servantList = []
        def mapper = new ObjectMapper()
        logger.info("开始转换JSON")
        def featureStr = "var datadetail = "
        for (def id : idList) {
            def url = baseUrl + "/guide/petdetail/" + id
            def html = jedis.hget("pages", url)
            def startAt = html.indexOf(featureStr)
            def endAt = html.indexOf("\n", startAt)
            def jsonStr = html.substring(startAt + featureStr.length() + 1, endAt - 2)
            ServantModel servantMdl = mapper.readValue(jsonStr, ServantModel.class)
            servantList.add(servantMdl.toServant())
        }
        logger.info("JSON转换完成")
        return servantList
    }
}
